{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/data/users/fyx/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/data/users/fyx/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matchzoo as mz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pack = mz.datasets.wiki_qa.load_data('train', task='ranking')\n",
    "valid_pack = mz.datasets.wiki_qa.load_data('dev', task='ranking', filter=True)\n",
    "predict_pack = mz.datasets.wiki_qa.load_data('test', task='ranking', filter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 2118/2118 [00:00<00:00, 8484.40it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 18841/18841 [00:03<00:00, 4813.45it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 745292.76it/s]\n",
      "Building FrequencyFilterUnit from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 128923.81it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 74693.13it/s] \n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 610426.43it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 660903.41it/s]\n",
      "Building VocabularyUnit from a datapack.: 100%|██████████| 404415/404415 [00:00<00:00, 2738016.34it/s]\n",
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 2118/2118 [00:00<00:00, 8982.45it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 18841/18841 [00:03<00:00, 4811.44it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 123979.66it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 167496.95it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 121546.26it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 583828.59it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 713633.16it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 121366.41it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 92625.66it/s]\n"
     ]
    }
   ],
   "source": [
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=10, fixed_length_right=40, remove_stop_words=False)\n",
    "train_pack_processed = preprocessor.fit_transform(train_pack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 122/122 [00:00<00:00, 8548.65it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 1115/1115 [00:00<00:00, 4881.14it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 122560.12it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 121516.29it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 124200.59it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 143656.68it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 550647.47it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 75528.43it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 75038.89it/s]\n",
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 237/237 [00:00<00:00, 9137.92it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 2300/2300 [00:00<00:00, 4848.27it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 133001.97it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 149706.33it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 132254.38it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 240515.38it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 581558.91it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 95692.15it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 92696.25it/s]\n"
     ]
    }
   ],
   "source": [
    "valid_pack_processed = preprocessor.transform(valid_pack)\n",
    "predict_pack_processed = preprocessor.transform(predict_pack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_task = mz.tasks.Ranking(loss=mz.losses.RankHingeLoss())\n",
    "ranking_task.metrics = [\n",
    "    mz.metrics.NormalizedDiscountedCumulativeGain(k=3),\n",
    "    mz.metrics.NormalizedDiscountedCumulativeGain(k=5),\n",
    "    mz.metrics.MeanAveragePrecision()\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter \"name\" set to DUET.\n",
      "Parameter \"embedding_trainable\" set to True.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           multiple             5002200     text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 10, 32)       28832       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 10, 32)       0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 40, 32)       28832       embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 1, 32)        0           dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 40, 32)       0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)             (None, 32)           0           max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 10, 32)       0           dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 10, 40)       0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 32)           1056        reshape_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 10, 32)       1056        max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 10, 32)       51232       lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 1, 32)        0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 10, 32)       0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 10, 32)       0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 10, 32)       0           lambda_2[0][0]                   \n",
      "                                                                 dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 320)          0           dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "reshape_3 (Reshape)             (None, 320)          0           lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 32)           10272       reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 32)           10272       reshape_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 32)           0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 32)           0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            33          dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            33          dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 1)            0           dense_2[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1)            2           add_1[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 5,133,820\n",
      "Trainable params: 5,133,820\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.DUET()\n",
    "model.params['input_shapes'] = preprocessor.context['input_shapes']\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding_input_dim'] = preprocessor.context['vocab_size']\n",
    "model.params['embedding_output_dim'] = 300\n",
    "model.params['lm_filters'] = 32\n",
    "model.params['lm_hidden_sizes'] = [32]\n",
    "model.params['dm_filters'] = 32\n",
    "model.params['dm_kernel_size'] = 3\n",
    "model.params['dm_d_mpool'] = 4\n",
    "model.params['dm_hidden_sizes'] = [32]\n",
    "model.params['dropout_rate'] = 0.5\n",
    "model.params['optimizer'] = 'adagrad'\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "model.compile()\n",
    "model.backend.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=300)\n",
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_embedding_matrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_x, pred_y = predict_pack_processed[:].unpack()\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model, x=pred_x, y=pred_y, batch_size=len(pred_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_generator = mz.PairDataGenerator(train_pack_processed, num_dup=2, num_neg=1, batch_size=20)\n",
    "len(train_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "102/102 [==============================] - 7s 68ms/step - loss: 1.1611\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5285485508994274 - normalized_discounted_cumulative_gain@5(0): 0.6002174150475211 - mean_average_precision(0): 0.5483873824220945\n",
      "Epoch 2/30\n",
      "102/102 [==============================] - 6s 54ms/step - loss: 0.7425\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5539639184214037 - normalized_discounted_cumulative_gain@5(0): 0.6120134302500022 - mean_average_precision(0): 0.5739963779648841\n",
      "Epoch 3/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.6449\n",
      "\n",
      "Epoch 3/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5754291916974498 - normalized_discounted_cumulative_gain@5(0): 0.6458393893065979 - mean_average_precision(0): 0.5998328780890881\n",
      "Epoch 4/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.5448\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5993020241109742 - normalized_discounted_cumulative_gain@5(0): 0.6571922889523615 - mean_average_precision(0): 0.616213721229544\n",
      "Epoch 5/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.4554\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5990473830050274 - normalized_discounted_cumulative_gain@5(0): 0.6551261044060693 - mean_average_precision(0): 0.6135633749690882\n",
      "Epoch 6/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.4432\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5990473830050274 - normalized_discounted_cumulative_gain@5(0): 0.6551261044060693 - mean_average_precision(0): 0.6135633749690882\n",
      "Epoch 6/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5965216279597069 - normalized_discounted_cumulative_gain@5(0): 0.6499821576508987 - mean_average_precision(0): 0.615420332034075\n",
      "Epoch 7/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.3572\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5965216279597069 - normalized_discounted_cumulative_gain@5(0): 0.6499821576508987 - mean_average_precision(0): 0.615420332034075\n",
      "Epoch 7/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.604369510147193 - normalized_discounted_cumulative_gain@5(0): 0.6569006353611903 - mean_average_precision(0): 0.6123206586786258\n",
      "Epoch 8/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.2987\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5882172076372432 - normalized_discounted_cumulative_gain@5(0): 0.6497066957273662 - mean_average_precision(0): 0.604248004105599\n",
      "Epoch 9/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.2734\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5900142850826946 - normalized_discounted_cumulative_gain@5(0): 0.6463561954508604 - mean_average_precision(0): 0.6015890315732088\n",
      "Epoch 10/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.2405\n",
      "\n",
      "Epoch 10/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.6030464546537169 - normalized_discounted_cumulative_gain@5(0): 0.6505898564660069 - mean_average_precision(0): 0.6082222467615538\n",
      "Epoch 11/30\n",
      "102/102 [==============================] - 6s 62ms/step - loss: 0.2123\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.6030464546537169 - normalized_discounted_cumulative_gain@5(0): 0.6505898564660069 - mean_average_precision(0): 0.6082222467615538\n",
      "Epoch 11/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5989520608483673 - normalized_discounted_cumulative_gain@5(0): 0.6482561741089696 - mean_average_precision(0): 0.6053265547835834\n",
      "Epoch 12/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.1868\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5989520608483673 - normalized_discounted_cumulative_gain@5(0): 0.6482561741089696 - mean_average_precision(0): 0.6053265547835834\n",
      "Epoch 12/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5868553370572612 - normalized_discounted_cumulative_gain@5(0): 0.6353233628829033 - mean_average_precision(0): 0.5992231274659656\n",
      "Epoch 13/30\n",
      "102/102 [==============================] - 6s 62ms/step - loss: 0.1868\n",
      "\n",
      "Epoch 13/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5766501548492312 - normalized_discounted_cumulative_gain@5(0): 0.6259698826950305 - mean_average_precision(0): 0.5819159844240182\n",
      "Epoch 14/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.1335\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5766501548492312 - normalized_discounted_cumulative_gain@5(0): 0.6259698826950305 - mean_average_precision(0): 0.5819159844240182\n",
      "Epoch 14/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5804421333916815 - normalized_discounted_cumulative_gain@5(0): 0.6360339268917256 - mean_average_precision(0): 0.5928171092751323\n",
      "Epoch 15/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.1224\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5774264789854833 - normalized_discounted_cumulative_gain@5(0): 0.636982537541942 - mean_average_precision(0): 0.5900139781519828\n",
      "Epoch 16/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.1215\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5746335977888043 - normalized_discounted_cumulative_gain@5(0): 0.6282359802346235 - mean_average_precision(0): 0.5850735558382911\n",
      "Epoch 17/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.1075\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5746335977888043 - normalized_discounted_cumulative_gain@5(0): 0.6282359802346235 - mean_average_precision(0): 0.5850735558382911\n",
      "Epoch 17/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5836350745048057 - normalized_discounted_cumulative_gain@5(0): 0.6356642447624318 - mean_average_precision(0): 0.5972847843418687\n",
      "Epoch 18/30\n",
      "102/102 [==============================] - 6s 61ms/step - loss: 0.0877\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5836350745048057 - normalized_discounted_cumulative_gain@5(0): 0.6356642447624318 - mean_average_precision(0): 0.5972847843418687\n",
      "Epoch 18/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5794803721126185 - normalized_discounted_cumulative_gain@5(0): 0.6364570467583311 - mean_average_precision(0): 0.5915638219829052\n",
      "Epoch 19/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0954\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5917850964134728 - normalized_discounted_cumulative_gain@5(0): 0.6410146892111024 - mean_average_precision(0): 0.5961566564591315\n",
      "Epoch 20/30\n",
      "102/102 [==============================] - 6s 58ms/step - loss: 0.0827\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5917850964134728 - normalized_discounted_cumulative_gain@5(0): 0.6410146892111024 - mean_average_precision(0): 0.5961566564591315\n",
      "Epoch 20/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5840237395430239 - normalized_discounted_cumulative_gain@5(0): 0.6365610577216884 - mean_average_precision(0): 0.5883660584609952\n",
      "Epoch 21/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0736\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5840237395430239 - normalized_discounted_cumulative_gain@5(0): 0.6365610577216884 - mean_average_precision(0): 0.5883660584609952\n",
      "Epoch 21/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5824312702438299 - normalized_discounted_cumulative_gain@5(0): 0.6476899583234097 - mean_average_precision(0): 0.5953916446766462\n",
      "Epoch 22/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0704\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5824312702438299 - normalized_discounted_cumulative_gain@5(0): 0.6476899583234097 - mean_average_precision(0): 0.5953916446766462\n",
      "Epoch 22/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5837851383090757 - normalized_discounted_cumulative_gain@5(0): 0.64057082171049 - mean_average_precision(0): 0.5920210255626595\n",
      "Epoch 23/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0644\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5858505248511132 - normalized_discounted_cumulative_gain@5(0): 0.6448034790882025 - mean_average_precision(0): 0.6012140002360248\n",
      "Epoch 24/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.0708\n",
      "\n",
      "Epoch 24/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5828995773354282 - normalized_discounted_cumulative_gain@5(0): 0.6380952646106933 - mean_average_precision(0): 0.5924163408420962\n",
      "Epoch 25/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.0596\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5828995773354282 - normalized_discounted_cumulative_gain@5(0): 0.6380952646106933 - mean_average_precision(0): 0.5924163408420962\n",
      "Epoch 25/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5907564536681598 - normalized_discounted_cumulative_gain@5(0): 0.6457727504590735 - mean_average_precision(0): 0.6027660580655791\n",
      "Epoch 26/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0541\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5894675537254703 - normalized_discounted_cumulative_gain@5(0): 0.6374197511879395 - mean_average_precision(0): 0.6017548753840395\n",
      "Epoch 27/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.0599\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5842991437508368 - normalized_discounted_cumulative_gain@5(0): 0.6405518654336088 - mean_average_precision(0): 0.5988537829378249\n",
      "Epoch 28/30\n",
      "102/102 [==============================] - 6s 59ms/step - loss: 0.0362\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.584840047169062 - normalized_discounted_cumulative_gain@5(0): 0.6445483137506833 - mean_average_precision(0): 0.6003470804768958\n",
      "Epoch 29/30\n",
      "102/102 [==============================] - 6s 58ms/step - loss: 0.0477\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5747463749108397 - normalized_discounted_cumulative_gain@5(0): 0.6363780939162336 - mean_average_precision(0): 0.5913073811411438\n",
      "Epoch 30/30\n",
      "102/102 [==============================] - 6s 60ms/step - loss: 0.0437\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5747463749108397 - normalized_discounted_cumulative_gain@5(0): 0.6363780939162336 - mean_average_precision(0): 0.5913073811411438\n",
      "Epoch 30/30\n",
      "Validation: normalized_discounted_cumulative_gain@3(0): 0.5718896796852097 - normalized_discounted_cumulative_gain@5(0): 0.6342360652719005 - mean_average_precision(0): 0.5894982942630591\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(train_generator, epochs=30, callbacks=[evaluate], workers=30, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
